{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Origin_trn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": [],
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "origin_address='/data/DW/DataFusion/dfc2019-master/track4/trn_data_folder/origin_onlychangevoxelsize_pickle/'\n",
    "\n",
    "with open(os.path.join(origin_address, 'dfc_train_dataset.pickle'),'rb') as f:########处理后的数据\n",
    "    ori_trn_data = pickle.load(f)\n",
    "with open(os.path.join(origin_address, 'dfc_train_metadata.pickle'),'rb') as f:#######总体统计信息\n",
    "    ori_trn_meta = pickle.load(f)\n",
    "with open(os.path.join(origin_address, 'dfc_train_labels.pickle'),'rb') as f:#######对应label\n",
    "    ori_trn_label = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(56299, 5) (56299,)\n",
      "{'cls_hist': {0: 8444551, 17: 1710296, 2: 103685489, 5: 25722504, 6: 20762346, 9: 2119080}, 'decompress_label_map': {0: 0, 1: 2, 2: 5, 3: 6, 4: 9, 5: 17}, 'compressed_label_map': {0: 0, 17: 5, 2: 1, 5: 2, 6: 3, 9: 4}, 'variance': array([2.75692883e+03, 2.77633770e+03, 4.60301771e+01, 4.52507971e+03,\n",
      "       1.84483880e-01])}\n"
     ]
    }
   ],
   "source": [
    "print(ori_trn_data[0].shape,ori_trn_label[0].shape)\n",
    "print(ori_trn_meta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ***Origin txt***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_list_of_files():\n",
    "    \n",
    "    files = glob.glob(os.path.join('/data/Public Data/IEEE Data Fusion Contest 2019/DFC2019_track4_trainval/Train-Track4/',\"*_PC3.txt\"))\n",
    "    return sorted(files)\n",
    "\n",
    "files=get_list_of_files()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "110"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(files)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(481290, 5) (481290, 1)\n",
      "finishd numbers:0 (481290, 1)\n",
      "(541209, 5) (541209, 1)\n",
      "finishd numbers:1 (541209, 1)\n",
      "(434287, 5) (434287, 1)\n",
      "finishd numbers:2 (434287, 1)\n",
      "(490995, 5) (490995, 1)\n",
      "finishd numbers:3 (490995, 1)\n",
      "(434742, 5) (434742, 1)\n",
      "finishd numbers:4 (434742, 1)\n",
      "(518905, 5) (518905, 1)\n",
      "finishd numbers:5 (518905, 1)\n",
      "(509287, 5) (509287, 1)\n",
      "finishd numbers:6 (509287, 1)\n",
      "(487931, 5) (487931, 1)\n",
      "finishd numbers:7 (487931, 1)\n",
      "(385768, 5) (385768, 1)\n",
      "finishd numbers:8 (385768, 1)\n",
      "(500046, 5) (500046, 1)\n",
      "finishd numbers:9 (500046, 1)\n",
      "(396088, 5) (396088, 1)\n",
      "finishd numbers:10 (396088, 1)\n",
      "(454517, 5) (454517, 1)\n",
      "finishd numbers:11 (454517, 1)\n",
      "(503233, 5) (503233, 1)\n",
      "finishd numbers:12 (503233, 1)\n",
      "(471080, 5) (471080, 1)\n",
      "finishd numbers:13 (471080, 1)\n",
      "(483161, 5) (483161, 1)\n",
      "finishd numbers:14 (483161, 1)\n",
      "(403927, 5) (403927, 1)\n",
      "finishd numbers:15 (403927, 1)\n",
      "(384721, 5) (384721, 1)\n",
      "finishd numbers:16 (384721, 1)\n",
      "(484902, 5) (484902, 1)\n",
      "finishd numbers:17 (484902, 1)\n",
      "(482490, 5) (482490, 1)\n",
      "finishd numbers:18 (482490, 1)\n",
      "(346755, 5) (346755, 1)\n",
      "finishd numbers:19 (346755, 1)\n",
      "(465490, 5) (465490, 1)\n",
      "finishd numbers:20 (465490, 1)\n",
      "(365112, 5) (365112, 1)\n",
      "finishd numbers:21 (365112, 1)\n",
      "(357645, 5) (357645, 1)\n",
      "finishd numbers:22 (357645, 1)\n",
      "(352323, 5) (352323, 1)\n",
      "finishd numbers:23 (352323, 1)\n",
      "(403780, 5) (403780, 1)\n",
      "finishd numbers:24 (403780, 1)\n",
      "(390158, 5) (390158, 1)\n",
      "finishd numbers:25 (390158, 1)\n",
      "(353972, 5) (353972, 1)\n",
      "finishd numbers:26 (353972, 1)\n",
      "(396457, 5) (396457, 1)\n",
      "finishd numbers:27 (396457, 1)\n",
      "(400558, 5) (400558, 1)\n",
      "finishd numbers:28 (400558, 1)\n",
      "(238792, 5) (238792, 1)\n",
      "finishd numbers:29 (238792, 1)\n",
      "(1019790, 5) (1019790, 1)\n",
      "finishd numbers:30 (1019790, 1)\n",
      "(691370, 5) (691370, 1)\n",
      "finishd numbers:31 (691370, 1)\n",
      "(882171, 5) (882171, 1)\n",
      "finishd numbers:32 (882171, 1)\n",
      "(244882, 5) (244882, 1)\n",
      "finishd numbers:33 (244882, 1)\n",
      "(724007, 5) (724007, 1)\n",
      "finishd numbers:34 (724007, 1)\n",
      "(634890, 5) (634890, 1)\n",
      "finishd numbers:35 (634890, 1)\n",
      "(494015, 5) (494015, 1)\n",
      "finishd numbers:36 (494015, 1)\n",
      "(1009846, 5) (1009846, 1)\n",
      "finishd numbers:37 (1009846, 1)\n",
      "(873383, 5) (873383, 1)\n",
      "finishd numbers:38 (873383, 1)\n",
      "(721168, 5) (721168, 1)\n",
      "finishd numbers:39 (721168, 1)\n",
      "(793603, 5) (793603, 1)\n",
      "finishd numbers:40 (793603, 1)\n",
      "(821746, 5) (821746, 1)\n",
      "finishd numbers:41 (821746, 1)\n",
      "(868445, 5) (868445, 1)\n",
      "finishd numbers:42 (868445, 1)\n",
      "(914428, 5) (914428, 1)\n",
      "finishd numbers:43 (914428, 1)\n",
      "(772840, 5) (772840, 1)\n",
      "finishd numbers:44 (772840, 1)\n",
      "(535819, 5) (535819, 1)\n",
      "finishd numbers:45 (535819, 1)\n",
      "(382955, 5) (382955, 1)\n",
      "finishd numbers:46 (382955, 1)\n",
      "(554135, 5) (554135, 1)\n",
      "finishd numbers:47 (554135, 1)\n",
      "(497923, 5) (497923, 1)\n",
      "finishd numbers:48 (497923, 1)\n",
      "(571774, 5) (571774, 1)\n",
      "finishd numbers:49 (571774, 1)\n",
      "(436603, 5) (436603, 1)\n",
      "finishd numbers:50 (436603, 1)\n",
      "(362372, 5) (362372, 1)\n",
      "finishd numbers:51 (362372, 1)\n",
      "(522347, 5) (522347, 1)\n",
      "finishd numbers:52 (522347, 1)\n",
      "(440122, 5) (440122, 1)\n",
      "finishd numbers:53 (440122, 1)\n",
      "(451816, 5) (451816, 1)\n",
      "finishd numbers:54 (451816, 1)\n",
      "(1093301, 5) (1093301, 1)\n",
      "finishd numbers:55 (1093301, 1)\n",
      "(578213, 5) (578213, 1)\n",
      "finishd numbers:56 (578213, 1)\n",
      "(628691, 5) (628691, 1)\n",
      "finishd numbers:57 (628691, 1)\n",
      "(560878, 5) (560878, 1)\n",
      "finishd numbers:58 (560878, 1)\n",
      "(550325, 5) (550325, 1)\n",
      "finishd numbers:59 (550325, 1)\n",
      "(796378, 5) (796378, 1)\n",
      "finishd numbers:60 (796378, 1)\n",
      "(685112, 5) (685112, 1)\n",
      "finishd numbers:61 (685112, 1)\n",
      "(513620, 5) (513620, 1)\n",
      "finishd numbers:62 (513620, 1)\n",
      "(573352, 5) (573352, 1)\n",
      "finishd numbers:63 (573352, 1)\n",
      "(471001, 5) (471001, 1)\n",
      "finishd numbers:64 (471001, 1)\n",
      "(816892, 5) (816892, 1)\n",
      "finishd numbers:65 (816892, 1)\n",
      "(969104, 5) (969104, 1)\n",
      "finishd numbers:66 (969104, 1)\n",
      "(496539, 5) (496539, 1)\n",
      "finishd numbers:67 (496539, 1)\n",
      "(679875, 5) (679875, 1)\n",
      "finishd numbers:68 (679875, 1)\n",
      "(652845, 5) (652845, 1)\n",
      "finishd numbers:69 (652845, 1)\n",
      "(546969, 5) (546969, 1)\n",
      "finishd numbers:70 (546969, 1)\n",
      "(669840, 5) (669840, 1)\n",
      "finishd numbers:71 (669840, 1)\n",
      "(414085, 5) (414085, 1)\n",
      "finishd numbers:72 (414085, 1)\n",
      "(435586, 5) (435586, 1)\n",
      "finishd numbers:73 (435586, 1)\n",
      "(1288419, 5) (1288419, 1)\n",
      "finishd numbers:74 (1288419, 1)\n",
      "(1635792, 5) (1635792, 1)\n",
      "finishd numbers:75 (1635792, 1)\n",
      "(1498698, 5) (1498698, 1)\n",
      "finishd numbers:76 (1498698, 1)\n",
      "(1315794, 5) (1315794, 1)\n",
      "finishd numbers:77 (1315794, 1)\n",
      "(711119, 5) (711119, 1)\n",
      "finishd numbers:78 (711119, 1)\n",
      "(743571, 5) (743571, 1)\n",
      "finishd numbers:79 (743571, 1)\n",
      "(777163, 5) (777163, 1)\n",
      "finishd numbers:80 (777163, 1)\n",
      "(2278782, 5) (2278782, 1)\n",
      "finishd numbers:81 (2278782, 1)\n",
      "(2067946, 5) (2067946, 1)\n",
      "finishd numbers:82 (2067946, 1)\n",
      "(2211848, 5) (2211848, 1)\n",
      "finishd numbers:83 (2211848, 1)\n",
      "(1519625, 5) (1519625, 1)\n",
      "finishd numbers:84 (1519625, 1)\n",
      "(442070, 5) (442070, 1)\n",
      "finishd numbers:85 (442070, 1)\n",
      "(742657, 5) (742657, 1)\n",
      "finishd numbers:86 (742657, 1)\n",
      "(823560, 5) (823560, 1)\n",
      "finishd numbers:87 (823560, 1)\n",
      "(821992, 5) (821992, 1)\n",
      "finishd numbers:88 (821992, 1)\n",
      "(1612929, 5) (1612929, 1)\n",
      "finishd numbers:89 (1612929, 1)\n",
      "(2216872, 5) (2216872, 1)\n",
      "finishd numbers:90 (2216872, 1)\n",
      "(2178545, 5) (2178545, 1)\n",
      "finishd numbers:91 (2178545, 1)\n",
      "(1029695, 5) (1029695, 1)\n",
      "finishd numbers:92 (1029695, 1)\n",
      "(452731, 5) (452731, 1)\n",
      "finishd numbers:93 (452731, 1)\n",
      "(1760580, 5) (1760580, 1)\n",
      "finishd numbers:94 (1760580, 1)\n",
      "(2094262, 5) (2094262, 1)\n",
      "finishd numbers:95 (2094262, 1)\n",
      "(1336331, 5) (1336331, 1)\n",
      "finishd numbers:96 (1336331, 1)\n",
      "(716014, 5) (716014, 1)\n",
      "finishd numbers:97 (716014, 1)\n",
      "(901563, 5) (901563, 1)\n",
      "finishd numbers:98 (901563, 1)\n",
      "(535307, 5) (535307, 1)\n",
      "finishd numbers:99 (535307, 1)\n",
      "(1421022, 5) (1421022, 1)\n",
      "finishd numbers:100 (1421022, 1)\n",
      "(666650, 5) (666650, 1)\n",
      "finishd numbers:101 (666650, 1)\n",
      "(943494, 5) (943494, 1)\n",
      "finishd numbers:102 (943494, 1)\n",
      "(1294465, 5) (1294465, 1)\n",
      "finishd numbers:103 (1294465, 1)\n",
      "(303666, 5) (303666, 1)\n",
      "finishd numbers:104 (303666, 1)\n",
      "(621346, 5) (621346, 1)\n",
      "finishd numbers:105 (621346, 1)\n",
      "(937434, 5) (937434, 1)\n",
      "finishd numbers:106 (937434, 1)\n",
      "(881267, 5) (881267, 1)\n",
      "finishd numbers:107 (881267, 1)\n",
      "(770663, 5) (770663, 1)\n",
      "finishd numbers:108 (770663, 1)\n",
      "(825824, 5) (825824, 1)\n",
      "finishd numbers:109 (825824, 1)\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(files)):\n",
    "    origintxtdata=np.array(pd.read_csv(files[i],sep=',',header=None))\n",
    "    labelfile=files[i][:-7]+'CLS.txt'\n",
    "    origintxtlabel=np.array(pd.read_csv(labelfile,header=None))\n",
    "    print(origintxtdata.shape,origintxtlabel.shape)\n",
    "    \n",
    "    new_binlabel=origintxtlabel.copy()\n",
    "    new_binlabel[origintxtlabel[:,0]==0,:]=0\n",
    "    new_binlabel[origintxtlabel[:,0]==2,:]=1\n",
    "    new_binlabel[origintxtlabel[:,0]==5,:]=2\n",
    "    new_binlabel[origintxtlabel[:,0]==6,:]=2\n",
    "    new_binlabel[origintxtlabel[:,0]==9,:]=2\n",
    "    new_binlabel[origintxtlabel[:,0]==17,:]=2\n",
    "\n",
    "    print('finishd numbers:{}'.format(i),new_binlabel.shape)\n",
    "    \n",
    "    ###Output\n",
    "    path,datafilename=os.path.split(files[i])\n",
    "    outputdatapath='/data/DW/DataFusion/dfc2019-master/track4/trn_data_folder/two_stage/stage1/Train-TXT/'+datafilename\n",
    "    \n",
    "    dt_data=pd.DataFrame(origintxtdata)\n",
    "    dt_data.to_csv(outputdatapath,sep=',',header=None,index=False)\n",
    "    \n",
    "    path,labelfilename=os.path.split(labelfile)\n",
    "    outputlabelpath='/data/DW/DataFusion/dfc2019-master/track4/trn_data_folder/two_stage/stage1/Train-TXT/'+labelfilename\n",
    "    \n",
    "    dt_label=pd.DataFrame(new_binlabel)\n",
    "    dt_label.to_csv(outputlabelpath,sep=',',header=None,index=False)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Non_ground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(481290, 5) (481290, 1)\n",
      "finishd numbers:0 (195473, 5) (195473, 1)\n",
      "(541209, 5) (541209, 1)\n",
      "finishd numbers:1 (196674, 5) (196674, 1)\n",
      "(434287, 5) (434287, 1)\n",
      "finishd numbers:2 (159146, 5) (159146, 1)\n",
      "(490995, 5) (490995, 1)\n",
      "finishd numbers:3 (175137, 5) (175137, 1)\n",
      "(434742, 5) (434742, 1)\n",
      "finishd numbers:4 (178128, 5) (178128, 1)\n",
      "(518905, 5) (518905, 1)\n",
      "finishd numbers:5 (192893, 5) (192893, 1)\n",
      "(509287, 5) (509287, 1)\n",
      "finishd numbers:6 (224424, 5) (224424, 1)\n",
      "(487931, 5) (487931, 1)\n",
      "finishd numbers:7 (104031, 5) (104031, 1)\n",
      "(385768, 5) (385768, 1)\n",
      "finishd numbers:8 (120959, 5) (120959, 1)\n",
      "(500046, 5) (500046, 1)\n",
      "finishd numbers:9 (161398, 5) (161398, 1)\n",
      "(396088, 5) (396088, 1)\n",
      "finishd numbers:10 (138739, 5) (138739, 1)\n",
      "(454517, 5) (454517, 1)\n",
      "finishd numbers:11 (138107, 5) (138107, 1)\n",
      "(503233, 5) (503233, 1)\n",
      "finishd numbers:12 (179573, 5) (179573, 1)\n",
      "(471080, 5) (471080, 1)\n",
      "finishd numbers:13 (134414, 5) (134414, 1)\n",
      "(483161, 5) (483161, 1)\n",
      "finishd numbers:14 (179973, 5) (179973, 1)\n",
      "(403927, 5) (403927, 1)\n",
      "finishd numbers:15 (91500, 5) (91500, 1)\n",
      "(384721, 5) (384721, 1)\n",
      "finishd numbers:16 (109514, 5) (109514, 1)\n",
      "(484902, 5) (484902, 1)\n",
      "finishd numbers:17 (209064, 5) (209064, 1)\n",
      "(482490, 5) (482490, 1)\n",
      "finishd numbers:18 (208612, 5) (208612, 1)\n",
      "(346755, 5) (346755, 1)\n",
      "finishd numbers:19 (103288, 5) (103288, 1)\n",
      "(465490, 5) (465490, 1)\n",
      "finishd numbers:20 (162281, 5) (162281, 1)\n",
      "(365112, 5) (365112, 1)\n",
      "finishd numbers:21 (84859, 5) (84859, 1)\n",
      "(357645, 5) (357645, 1)\n",
      "finishd numbers:22 (85426, 5) (85426, 1)\n",
      "(352323, 5) (352323, 1)\n",
      "finishd numbers:23 (159454, 5) (159454, 1)\n",
      "(403780, 5) (403780, 1)\n",
      "finishd numbers:24 (170554, 5) (170554, 1)\n",
      "(390158, 5) (390158, 1)\n",
      "finishd numbers:25 (198765, 5) (198765, 1)\n",
      "(353972, 5) (353972, 1)\n",
      "finishd numbers:26 (95646, 5) (95646, 1)\n",
      "(396457, 5) (396457, 1)\n",
      "finishd numbers:27 (97565, 5) (97565, 1)\n",
      "(400558, 5) (400558, 1)\n",
      "finishd numbers:28 (100226, 5) (100226, 1)\n",
      "(238792, 5) (238792, 1)\n",
      "finishd numbers:29 (115459, 5) (115459, 1)\n",
      "(1019790, 5) (1019790, 1)\n",
      "finishd numbers:30 (473823, 5) (473823, 1)\n",
      "(691370, 5) (691370, 1)\n",
      "finishd numbers:31 (169373, 5) (169373, 1)\n",
      "(882171, 5) (882171, 1)\n",
      "finishd numbers:32 (322460, 5) (322460, 1)\n",
      "(244882, 5) (244882, 1)\n",
      "finishd numbers:33 (175793, 5) (175793, 1)\n",
      "(724007, 5) (724007, 1)\n",
      "finishd numbers:34 (358835, 5) (358835, 1)\n",
      "(634890, 5) (634890, 1)\n",
      "finishd numbers:35 (106188, 5) (106188, 1)\n",
      "(494015, 5) (494015, 1)\n",
      "finishd numbers:36 (289543, 5) (289543, 1)\n",
      "(1009846, 5) (1009846, 1)\n",
      "finishd numbers:37 (512336, 5) (512336, 1)\n",
      "(873383, 5) (873383, 1)\n",
      "finishd numbers:38 (526304, 5) (526304, 1)\n",
      "(721168, 5) (721168, 1)\n",
      "finishd numbers:39 (411160, 5) (411160, 1)\n",
      "(793603, 5) (793603, 1)\n",
      "finishd numbers:40 (255199, 5) (255199, 1)\n",
      "(821746, 5) (821746, 1)\n",
      "finishd numbers:41 (494031, 5) (494031, 1)\n",
      "(868445, 5) (868445, 1)\n",
      "finishd numbers:42 (572623, 5) (572623, 1)\n",
      "(914428, 5) (914428, 1)\n",
      "finishd numbers:43 (452279, 5) (452279, 1)\n",
      "(772840, 5) (772840, 1)\n",
      "finishd numbers:44 (428068, 5) (428068, 1)\n",
      "(535819, 5) (535819, 1)\n",
      "finishd numbers:45 (250547, 5) (250547, 1)\n",
      "(382955, 5) (382955, 1)\n",
      "finishd numbers:46 (203224, 5) (203224, 1)\n",
      "(554135, 5) (554135, 1)\n",
      "finishd numbers:47 (255665, 5) (255665, 1)\n",
      "(497923, 5) (497923, 1)\n",
      "finishd numbers:48 (188095, 5) (188095, 1)\n",
      "(571774, 5) (571774, 1)\n",
      "finishd numbers:49 (231303, 5) (231303, 1)\n",
      "(436603, 5) (436603, 1)\n",
      "finishd numbers:50 (184528, 5) (184528, 1)\n",
      "(362372, 5) (362372, 1)\n",
      "finishd numbers:51 (93241, 5) (93241, 1)\n",
      "(522347, 5) (522347, 1)\n",
      "finishd numbers:52 (254111, 5) (254111, 1)\n",
      "(440122, 5) (440122, 1)\n",
      "finishd numbers:53 (236247, 5) (236247, 1)\n",
      "(451816, 5) (451816, 1)\n",
      "finishd numbers:54 (124164, 5) (124164, 1)\n",
      "(1093301, 5) (1093301, 1)\n",
      "finishd numbers:55 (356412, 5) (356412, 1)\n",
      "(578213, 5) (578213, 1)\n",
      "finishd numbers:56 (0, 5) (0, 1)\n",
      "(628691, 5) (628691, 1)\n",
      "finishd numbers:57 (78053, 5) (78053, 1)\n",
      "(560878, 5) (560878, 1)\n",
      "finishd numbers:58 (115341, 5) (115341, 1)\n",
      "(550325, 5) (550325, 1)\n",
      "finishd numbers:59 (155697, 5) (155697, 1)\n",
      "(796378, 5) (796378, 1)\n",
      "finishd numbers:60 (251469, 5) (251469, 1)\n",
      "(685112, 5) (685112, 1)\n",
      "finishd numbers:61 (263660, 5) (263660, 1)\n",
      "(513620, 5) (513620, 1)\n",
      "finishd numbers:62 (127492, 5) (127492, 1)\n",
      "(573352, 5) (573352, 1)\n",
      "finishd numbers:63 (83806, 5) (83806, 1)\n",
      "(471001, 5) (471001, 1)\n",
      "finishd numbers:64 (144094, 5) (144094, 1)\n",
      "(816892, 5) (816892, 1)\n",
      "finishd numbers:65 (241571, 5) (241571, 1)\n",
      "(969104, 5) (969104, 1)\n",
      "finishd numbers:66 (1811, 5) (1811, 1)\n",
      "(496539, 5) (496539, 1)\n",
      "finishd numbers:67 (0, 5) (0, 1)\n",
      "(679875, 5) (679875, 1)\n",
      "finishd numbers:68 (174842, 5) (174842, 1)\n",
      "(652845, 5) (652845, 1)\n",
      "finishd numbers:69 (38236, 5) (38236, 1)\n",
      "(546969, 5) (546969, 1)\n",
      "finishd numbers:70 (124245, 5) (124245, 1)\n",
      "(669840, 5) (669840, 1)\n",
      "finishd numbers:71 (177036, 5) (177036, 1)\n",
      "(414085, 5) (414085, 1)\n",
      "finishd numbers:72 (63438, 5) (63438, 1)\n",
      "(435586, 5) (435586, 1)\n",
      "finishd numbers:73 (1940, 5) (1940, 1)\n",
      "(1288419, 5) (1288419, 1)\n",
      "finishd numbers:74 (521791, 5) (521791, 1)\n",
      "(1635792, 5) (1635792, 1)\n",
      "finishd numbers:75 (309711, 5) (309711, 1)\n",
      "(1498698, 5) (1498698, 1)\n",
      "finishd numbers:76 (181434, 5) (181434, 1)\n",
      "(1315794, 5) (1315794, 1)\n",
      "finishd numbers:77 (127684, 5) (127684, 1)\n",
      "(711119, 5) (711119, 1)\n",
      "finishd numbers:78 (226450, 5) (226450, 1)\n",
      "(743571, 5) (743571, 1)\n",
      "finishd numbers:79 (127973, 5) (127973, 1)\n",
      "(777163, 5) (777163, 1)\n",
      "finishd numbers:80 (173942, 5) (173942, 1)\n",
      "(2278782, 5) (2278782, 1)\n",
      "finishd numbers:81 (595643, 5) (595643, 1)\n",
      "(2067946, 5) (2067946, 1)\n",
      "finishd numbers:82 (547537, 5) (547537, 1)\n",
      "(2211848, 5) (2211848, 1)\n",
      "finishd numbers:83 (557129, 5) (557129, 1)\n",
      "(1519625, 5) (1519625, 1)\n",
      "finishd numbers:84 (447512, 5) (447512, 1)\n",
      "(442070, 5) (442070, 1)\n",
      "finishd numbers:85 (65919, 5) (65919, 1)\n",
      "(742657, 5) (742657, 1)\n",
      "finishd numbers:86 (138677, 5) (138677, 1)\n",
      "(823560, 5) (823560, 1)\n",
      "finishd numbers:87 (341073, 5) (341073, 1)\n",
      "(821992, 5) (821992, 1)\n",
      "finishd numbers:88 (353488, 5) (353488, 1)\n",
      "(1612929, 5) (1612929, 1)\n",
      "finishd numbers:89 (387458, 5) (387458, 1)\n",
      "(2216872, 5) (2216872, 1)\n",
      "finishd numbers:90 (805183, 5) (805183, 1)\n",
      "(2178545, 5) (2178545, 1)\n",
      "finishd numbers:91 (807409, 5) (807409, 1)\n",
      "(1029695, 5) (1029695, 1)\n",
      "finishd numbers:92 (291343, 5) (291343, 1)\n",
      "(452731, 5) (452731, 1)\n",
      "finishd numbers:93 (93947, 5) (93947, 1)\n",
      "(1760580, 5) (1760580, 1)\n",
      "finishd numbers:94 (496802, 5) (496802, 1)\n",
      "(2094262, 5) (2094262, 1)\n",
      "finishd numbers:95 (677226, 5) (677226, 1)\n",
      "(1336331, 5) (1336331, 1)\n",
      "finishd numbers:96 (179298, 5) (179298, 1)\n",
      "(716014, 5) (716014, 1)\n",
      "finishd numbers:97 (128875, 5) (128875, 1)\n",
      "(901563, 5) (901563, 1)\n",
      "finishd numbers:98 (189345, 5) (189345, 1)\n",
      "(535307, 5) (535307, 1)\n",
      "finishd numbers:99 (124093, 5) (124093, 1)\n",
      "(1421022, 5) (1421022, 1)\n",
      "finishd numbers:100 (630594, 5) (630594, 1)\n",
      "(666650, 5) (666650, 1)\n",
      "finishd numbers:101 (135057, 5) (135057, 1)\n",
      "(943494, 5) (943494, 1)\n",
      "finishd numbers:102 (333754, 5) (333754, 1)\n",
      "(1294465, 5) (1294465, 1)\n",
      "finishd numbers:103 (466442, 5) (466442, 1)\n",
      "(303666, 5) (303666, 1)\n",
      "finishd numbers:104 (38214, 5) (38214, 1)\n",
      "(621346, 5) (621346, 1)\n",
      "finishd numbers:105 (95841, 5) (95841, 1)\n",
      "(937434, 5) (937434, 1)\n",
      "finishd numbers:106 (39696, 5) (39696, 1)\n",
      "(881267, 5) (881267, 1)\n",
      "finishd numbers:107 (99975, 5) (99975, 1)\n",
      "(770663, 5) (770663, 1)\n",
      "finishd numbers:108 (138275, 5) (138275, 1)\n",
      "(825824, 5) (825824, 1)\n",
      "finishd numbers:109 (170395, 5) (170395, 1)\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(files)):\n",
    "    origintxtdata=np.array(pd.read_csv(files[i],sep=',',header=None))\n",
    "    labelfile=files[i][:-7]+'CLS.txt'\n",
    "    origintxtlabel=np.array(pd.read_csv(labelfile,header=None))\n",
    "    print(origintxtdata.shape,origintxtlabel.shape)\n",
    "    \n",
    "    new_nonglabel=origintxtlabel[origintxtlabel[:,0]>2,:]\n",
    "    new_nongdata=origintxtdata[origintxtlabel[:,0]>2,:]\n",
    "\n",
    "    print('finishd numbers:{}'.format(i),new_nongdata.shape,new_nonglabel.shape)\n",
    "    \n",
    "    ###Output\n",
    "    path,datafilename=os.path.split(files[i])\n",
    "    outputdatapath='/data/DW/DataFusion/dfc2019-master/track4/trn_data_folder/two_stage/stage2/Train-TXT/'+datafilename\n",
    "    \n",
    "    dt_data=pd.DataFrame(new_nongdata)\n",
    "    dt_data.to_csv(outputdatapath,sep=',',header=None,index=False)\n",
    "    \n",
    "    path,labelfilename=os.path.split(labelfile)\n",
    "    outputlabelpath='/data/DW/DataFusion/dfc2019-master/track4/trn_data_folder/two_stage/stage2/Train-TXT/'+labelfilename\n",
    "    \n",
    "    dt_label=pd.DataFrame(new_nonglabel)\n",
    "    dt_label.to_csv(outputlabelpath,sep=',',header=None,index=False)\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (py4torch_tf)",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
